package hou.etl

import hou.config.ConfigHelper
import hou.utils.JedisPools
import org.apache.spark.{SparkConf, SparkContext}

//字典数据放入到redis
object DictToRedis {
  def main(args: Array[String]): Unit = {
    //sparkContet
    val conf = new SparkConf()
    conf.setMaster("local[*]")
    conf.setAppName(s"${this.getClass.getName}")
    conf.set("spark.serializer", ConfigHelper.serializer)
    val sc = new SparkContext(conf)

    val appDict = sc.textFile("C:\\Users\\44323\\Desktop\\资料PDF\\app_dict.txt")
    val mapDict = appDict.map(_.split("\t",-1)).filter(_.length>=5)
      .foreachPartition(partition=>{
        val jedis =  JedisPools.getRedis(1)
        partition.foreach(arr=>{
          jedis.hset("appdict",arr(4),arr(1))
        })
        jedis.close()
      })

    sc.stop()
  }
}
